var express = require('express'),
	async = require('async'),
	superagent = require('superagent'),
	cheerio = require('cheerio'),
	url = require('url');

var app = express();
var cnodeUrl = 'https://cnodejs.org/';

var common = require('./common/common')

app.get('/',function(req,res,next){
	superagent.get(cnodeUrl)
		.end(function(err,sres){
			if(err){
				console.log(err);
				return next(err);
			}

			var topicUrls = [];
			var $ = cheerio.load(sres.text);

			$('#topic_list .topic_title').each(function(i,el){
				var $el = $(el);
				var href = url.resolve(cnodeUrl,$el.attr('href'));
				topicUrls.push(href);
			});
			// console.log(topicUrls)

			async.mapLimit(topicUrls,5,function(_url,callback){
					common.fetchUrl(_url,callback);
				},function(err,result){
					if(err) return console.log(err);
					res.send(result);
				});
		});
});


app.listen(3000,function(){
	console.log('app is running at port 3000');
});

































